# coding = utf8
__author__ = 'jiangchanghao'
from FavProcess import FavProcess
import math
from CategoryData import CategoryPaterns

class FavStat(FavProcess):

    def __init__(self, filename):
        FavProcess.__init__(self, filename)
        self.f_out = False

    def set_outfile_name(self, name):
        self.f_out = open(name, 'w')
        self.f_out.write(str(self.m_filename) + '\n')

    def close_file(self):
        if self.f_out:
            self.f_out.close()

class FavStat_User(FavStat):
    user_info = {} # user_info [user_id] [ folder name ] = ( url_total_number, [(title, url)] )

    def process_line(self, line, need_print=True):
        ss = line.split('\t')
        num = len(ss)
        assert num > 0 and (num - 2) % 3 == 0, "error for taking line syntax"
        user_id = ss[0]
        time_mod = ss[-1]
        url_count = 0
        t_user_info = {} # user_info[ folder name ] = ( url_total_number, [(title, url)] )
        idx = 1
        while idx < num - 1:
            url, title, tag = ss[idx:idx+3]
            url = self.url_refactor(url) # refactor the url first
            idx += 3
            # put information
            url_count += 1
            if not t_user_info.has_key(tag):
                t_user_info[tag] = [0,[]]
            t_user_info[tag][1].append( (title, url) )
            t_user_info[tag][0] += 1
        t_user_info[tag][1].sort(key=lambda x:x[1], reverse=True)
        self.user_info[user_id] = t_user_info
        if (need_print):
            # start to stat
            entropy_on_user_tag = 0
            tag_num = 0
            for tag in t_user_info.keys():
                p = 1.0 * t_user_info[tag][0] / url_count
                entropy_on_user_tag += p * math.log10(p)
                tag_num += 1
            self.f_out.write(user_id + '\t' + str(url_count) + '\t' + str(tag_num) + '\t' + str(entropy_on_user_tag) )
            self.f_out.write('\t')
            sorted_info = sorted(t_user_info.items(), key=lambda e:e[1][0], reverse=True)
            for item in sorted_info:
                self.f_out.write(' ' + item[0] + '[' + str(item[1][0]) + ']')
            self.f_out.write('\n')


class FavStat_Url(FavStat):
    url_info = {} # url_info = {[url]:{[tag]:[user_id]}}
    url_count = {} # url_count = {[url]:fav number}

    def process_line(self, line):
        ss = line.split('\t')
        num = len(ss)
        assert num > 0 and (num - 2) % 3 == 0, "error for taking line syntax"
        user_id = ss[0]
        time_mod = ss[-1]
        idx = 1
        while idx < num - 1:
            url, title, tag = ss[idx:idx+3]
            url = self.url_refactor(url) # refactor the url first
            idx += 3
            # put information
            if not self.url_info.has_key(url):
                self.url_info[url] = {}
                self.url_count[url] = 0
            if not self.url_info[url].has_key(tag):
                self.url_info[url][tag] = [user_id]
            else:
                self.url_info[url][tag].append(user_id)
            self.url_count[url] += 1

    def close_file(self):
        # print all information here
        for url in self.url_count.keys():
            count = self.url_count[url]
            if count < 5:
                continue
            tag_dict = sorted(self.url_info[url].items(), key=lambda e:len(e[1]), reverse=True) # tag_dict = [(tag, [user_id1, user_id2, user_id3, ...]) ]
            tag_entropy = 0
            for (tag, user_list) in tag_dict:
                p = 1.0 * len(user_list) / count
                tag_entropy += p * math.log10(p)
            self.f_out.write(url + '\t' + str(count) + '\t' + str(len(tag_dict)) + '\t' + str(tag_entropy) + '\t')
            for (tag, user_list) in tag_dict:
                user_num = len(user_list)
                if user_num > 5:
                    self.f_out.write(tag + '[' + str(user_num) + ']' + ' ')
                else:
                    self.f_out.write(tag + '[')
                    for user_id in user_list:
                        self.f_out.write(user_id + ',')
                    self.f_out.write('] ')
            self.f_out.write('\n')
        self.f_out.close()

class FavStatForPatterns(FavStat):
    def __init__(self, filename=''):
        FavStat.__init__(self, filename)
        patternSource = CategoryPaterns()
        patternSource.init()
        self.m_patterns = patternSource.patterns
        sz = len(self.m_patterns)
        self.pattern_match_info = [{} for i in range(0, sz)]  # {pattern_index: {url:number}}
        self.label_match_info = {}    # {tag: {label:{url:number} } }
        self.m_match_number = 0
        self.m_total_number = 0
        self.type_stat = {0:0, 1:0, 10:0, 11:0}

    def process_line(self, line):
        ss = line.split('\t')
        num = len(ss)
        assert num > 0 and (num - 2) % 3 == 0, "error for taking line syntax"
        user_id = ss[0]
        time_mod = ss[-1]
        idx = 1
        while idx < num - 1:
            url, title, tag = ss[idx:idx+3]
            #url = self.url_short(url) # refactor the url first
            idx += 3
            self.check_pattern(user_id, url, tag)

    def check_pattern(self, user_id, url, tag):
        match = False # decide whether cound or not
        new = False  # decide whether print or not
        for i in range(0, len(self.m_patterns)):
            (pattern, label) = self.m_patterns[i]
            type = 0
            if pattern.match(url):
                type += 1
                if not self.pattern_match_info[i].has_key(url):  # [pattern_index: {url:number}]
                    self.pattern_match_info[i][url] = 1
                    new = True
                else:
                    self.pattern_match_info[i][url] += 1
            if label in tag:
                type += 10
                if not self.label_match_info.has_key(label):
                    self.label_match_info[label] = {}
                if not self.label_match_info[label].has_key(tag):   # {label: {tag:{url:number} } }
                    self.label_match_info[label][tag] = {}
                    self.label_match_info[label][tag][url] = 1
                    new = True
                else:
                    if not self.label_match_info[label][tag].has_key(url):
                        self.label_match_info[label][tag][url] = 1
                    else:
                        self.label_match_info[label][tag][url] += 1
            if type > 0:
                match = True
                out_str = str(type) + '\t' + str(i) + '\t' + url + '\t' + label +  '\t'+ tag + '\t' + user_id + '\t' + self.m_current_parsing_filename + '\t' + pattern.pattern + '\n'
                if new:
                    self.f_out.write(out_str)
                break
        if match:
            self.m_match_number += 1
        self.m_total_number += 1
        self.type_stat[type] += 1
        if self.m_total_number % 1000 == 0:
            print '\t pattern checking ', self.m_total_number

    def close_file(self):
        self.f_out.write('--------MATCH URL---------:' + str(self.m_match_number) + '---------\n')
        self.f_out.write('--------TOTAL URL---------:' + str(self.m_total_number) + '---------\n')
        self.f_out.write('--------PATTERN_MATCH_INFO---------\n')
        for i in range(0, len(self.pattern_match_info)):# [pattern_index: {url:number}]
            (pt, label) = self.m_patterns[i]
            pattern = pt.pattern
            self.f_out.write(pattern + '\n')
            for url in self.pattern_match_info[i].keys():
                self.f_out.write('\t' + url + '\t' + str(self.pattern_match_info[i][url]) + '\n')

        self.f_out.write('--------LABEL_MATCH_INFO---------\n')
        for label in self.label_match_info.keys():# [tag_index: {tag:{url:number} } ]
            self.f_out.write(label + '\n')
            for tag in self.label_match_info[label].keys():
                url_info = self.label_match_info[label][tag]
                self.f_out.write('\t' + tag + '\n')
                for url in self.label_match_info[label][tag].keys():
                    self.f_out.write('\t\t' + url + '\t' + str(self.label_match_info[label][tag][url]) + '\n')

        self.f_out.write('--------TYPE_STAT---------\n')
        for type in self.type_stat.keys():
            self.f_out.write(str(type) + ':\t' + str(self.type_stat[type]) + '\n')
        self.f_out.close()

    def url_base(self, url):
        url = FavProcess.url_short(url)
        index = url.find('/')
        if index > 0:
            url = url[:index]
        return url



if __name__ == '__main__':
    #test = FavStat_User()
    #test = FavStat_Url()
    test = FavStatForPatterns()
    test.set_outfile_name('res/stat_url_2.txt')
    test.init()
    test.close_file()